
/*
* Copyright (c) Z.G. Shi  <blueflycn at gmail dot com>
* Make a better world~
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <dlfcn.h>
#include <time.h>
#include <stdint.h>
#include <fcntl.h>
#include <sys/stat.h>

#include "tyn_lib.h"
#include "tyn_binary_heap.h"
#include "tyn_exsorter.h"
#include "tyn_indexer.h"
#include "tyn_coder.h"
#include "uthash.h"
#include "qsort.h"
#include "sha1sum.h"


int tyn_cmp_attr32(attr32_t *a, attr32_t *b) {
    return (a->document_id > b->document_id);
}

int tyn_cmp_node_attr32(tyn_exsorter_node_t *a, tyn_exsorter_node_t *b) {
    return tyn_cmp_attr32((attr32_t *)b->data, (attr32_t *)a->data);
}

int tyn_cmp_term32(term32_t *a, term32_t *b) {
    return (a->word_id > b->word_id) || (a->word_id == b->word_id && a->document_id > b->document_id) || (a->word_id == b->word_id && a->document_id == b->document_id && a->field_id > b->field_id) || (a->word_id == b->word_id && a->document_id == b->document_id && a->field_id == b->field_id && a->pos > b->pos);
}

int tyn_cmp_node_term32(tyn_exsorter_node_t *a, tyn_exsorter_node_t *b) {
    return tyn_cmp_term32((term32_t *)b->data, (term32_t *)a->data);
}

int tyn_cmp_term64(term64_t *a, term64_t *b) {
    return (a->word_id > b->word_id) || (a->word_id == b->word_id && a->document_id > b->document_id) || (a->word_id == b->word_id && a->document_id == b->document_id && a->field_id > b->field_id) || (a->word_id == b->word_id && a->document_id == b->document_id && a->field_id == b->field_id && a->pos > b->pos);
}

int tyn_cmp_node_term64(tyn_exsorter_node_t *a, tyn_exsorter_node_t *b) {
    return tyn_cmp_term64((term32_t *)b->data, (term64_t *)a->data);
}

int tyn_sort_term32(void *array, size_t count, size_t size) {
    #define term_lt(a,b) (((a)->word_id < (b)->word_id) || ((a)->word_id == (b)->word_id && (a)->document_id < (b)->document_id) || ((a)->word_id == (b)->word_id && (a)->document_id == (b)->document_id && (a)->field_id < (b)->field_id) || ((a)->word_id == (b)->word_id && (a)->document_id == (b)->document_id && (a)->field_id == (b)->field_id && (a)->pos < (b)->pos))
    QSORT(term32_t, array, count, term_lt);
}

int tyn_sort_attr32(void *array, size_t count, size_t size) {
    #define attr_num_lt(a,b) ((a)->document_id < (b)->document_id)
    QSORT(attr32_t, array, count, attr_num_lt);
}



int tyn_term32_entity_sort_by_name(term32_entity_t *a, term32_entity_t *b) {
  return strcmp(a->str, b->str);
}

int progress_callback(size_t num_item_processed, size_t num_total_items, int is_last_call) {
    printf("sorted %.1f% \r", (float)num_item_processed *100 / (float)num_total_items);
    if( is_last_call) {
        printf("\nsorted %d, total %d\n", num_item_processed, num_total_items);
    }
    fflush(stdout);
}

int nodes_term32_flush(void *array, size_t size, void *nodes_context, int is_last) {
    term32_context_t *context = (term32_context_t *)nodes_context;
    
    term32_t *term32 = (term32_t *)array;
    size_t total = size / sizeof(term32_t);
    size_t counter = 0;
    
    if(context->is_first) {
            term32_entity_t *term32_entities = context->term32_entities;
            term32_entity_t *term32_entity = NULL;
            
            HASH_FIND(hh2,term32_entities,&term32->word_id,sizeof(uint32_t),term32_entity);
            if(term32_entity) {
                //printf("processing %s\n", term32_entity->str);
                //word_id = term32_entity->id;
                term32_entity->index.num_encoded_doc = 0;
                term32_entity->index.num_encoded_freq = 0;
                term32_entity->index.num_encoded_pos = 0;
                term32_entity->index.size_encoded_doc = 0;
                term32_entity->index.size_encoded_freq = 0;
                term32_entity->index.size_encoded_pos = 0;
                
                //term32_entity->index.offset_doc = context->offset_doc;
                //term32_entity->index.offset_freq = context->offset_freq;
                //term32_entity->index.offset_pos = context->offset_pos;
                context->term32_entity = term32_entity;
            } else {
                tyn_log(stderr, "fatal error: unknown word id");
            }
        memset(context->buffer_freq, 0, context->num_max_freq * sizeof(uint32_t));
        context->last_word_id = term32->word_id;
        context->is_first = 0;
    }
    
    while(counter < total) {
        //printf("w:%u\t d:%u\t f:%u\t p:%u\n", term32->word_id, term32->document_id, term32->field_id, term32->pos);
        //if(term32->word_id != context->last_word_id && !context->is_first) {
        if(term32->word_id != context->last_word_id) {
             context->num_wordid++;

            //----------------------------------------------------------------------------------
            //flush the buffer with termination tag
            size_t encoded_size;
            encoded_size = tyn_p4d_encode32(context->buffer_doc, context->num_doc, context->buffer_encode, 1);
            if(context->buffer_write_ptr_doc + encoded_size > context->buffer_write_max_doc) {
                //flush to disk
                tyn_write(context->fd_doc, context->buffer_write_doc, context->buffer_write_ptr_doc - context->buffer_write_doc);
                context->buffer_write_ptr_doc = context->buffer_write_doc;
            }
            memcpy(context->buffer_write_ptr_doc, context->buffer_encode, encoded_size);
            context->buffer_write_ptr_doc += encoded_size;
            context->term32_entity->index.size_encoded_doc += encoded_size;
            context->term32_entity->index.num_encoded_doc += context->num_doc;
            //write(context->fd_doc, context->buffer_write, encoded_size);
            context->doc_ptr = context->buffer_doc;
            //if( encoded_size > context->num_doc * sizeof(uint32_t)) printf("error encoding doc ids %d, %d\n", encoded_size,context->num_doc * sizeof(uint32_t) );
            context->num_doc = 0;
            //----------------------------------------------------------------------------------
            
            encoded_size = tyn_p4d_encode32(context->buffer_freq, context->num_freq, context->buffer_encode, 1);
            if(context->buffer_write_ptr_freq + encoded_size > context->buffer_write_max_freq) {
                //flush to disk
                tyn_write(context->fd_freq, context->buffer_write_freq, context->buffer_write_ptr_freq - context->buffer_write_freq);
                context->buffer_write_ptr_freq = context->buffer_write_freq;
            }
            memcpy(context->buffer_write_ptr_freq, context->buffer_encode, encoded_size);
            context->buffer_write_ptr_freq += encoded_size;
            context->term32_entity->index.size_encoded_freq += encoded_size;
            context->term32_entity->index.num_encoded_freq += context->num_freq;
                
            //write(context->fd_freq, context->buffer_write, encoded_size);
            memset(context->buffer_freq, 0, context->num_freq * sizeof(uint32_t));
            context->freq_ptr = context->buffer_freq;
            context->num_freq = 0;
            
            
            //----------------------------------------------------------------------------------
            encoded_size = tyn_p4d_encode32(context->buffer_pos, context->num_pos, context->buffer_encode, 1);
            if(context->buffer_write_ptr_pos + encoded_size > context->buffer_write_max_pos) {
                //flush to disk
                tyn_write(context->fd_pos, context->buffer_write_pos, context->buffer_write_ptr_pos - context->buffer_write_pos);
                context->buffer_write_ptr_pos = context->buffer_write_pos;
            }
            memcpy(context->buffer_write_ptr_pos, context->buffer_encode, encoded_size);
            context->buffer_write_ptr_pos += encoded_size;
            context->term32_entity->index.size_encoded_pos += encoded_size;
            context->term32_entity->index.num_encoded_pos += context->num_pos;
            
            //write(context->fd_pos, context->buffer_write, encoded_size);
            context->pos_ptr = context->buffer_pos;
            context->num_pos = 0;
            
            //---------------------------------------------------------------------------------
            //process term index

            //context->offset_doc += context->term32_entity->index.size_encoded_doc;
            //context->offset_freq += context->term32_entity->index.size_encoded_freq;
            //context->offset_pos += context->term32_entity->index.size_encoded_pos;
            
            term32_entity_t *term32_entities = context->term32_entities;
            term32_entity_t *term32_entity = NULL;
            
            HASH_FIND(hh2,term32_entities,&term32->word_id,sizeof(uint32_t),term32_entity);
            if(term32_entity) {
                //printf("processing %s\n", term32_entity->str);
                //word_id = term32_entity->id;
                term32_entity->index.num_encoded_doc = 0;
                term32_entity->index.num_encoded_freq = 0;
                term32_entity->index.num_encoded_pos = 0;
                term32_entity->index.size_encoded_doc = 0;
                term32_entity->index.size_encoded_freq = 0;
                term32_entity->index.size_encoded_pos = 0;
                
                //term32_entity->index.offset_doc = context->offset_doc;
                //term32_entity->index.offset_freq = context->offset_freq;
                //term32_entity->index.offset_pos = context->offset_pos;
                context->term32_entity = term32_entity;
            } else {
                tyn_log(stderr, "fatal error: unknown word id");
            }
            //-------------------------------------------------------------------------------------
            //context->is_first = 1;
            context->last_word_id = term32->word_id;
            context->last_document_id = 0;
        }
        if(term32->document_id != context->last_document_id) {
            if(context->num_doc == context->num_max_doc) {
                size_t encoded_size = tyn_p4d_encode32(context->buffer_doc, context->num_doc, context->buffer_encode, 0);
                if(context->buffer_write_ptr_doc + encoded_size > context->buffer_write_max_doc) {
                    //flush to disk
                    tyn_write(context->fd_doc, context->buffer_write_doc, context->buffer_write_ptr_doc - context->buffer_write_doc);
                    context->buffer_write_ptr_doc = context->buffer_write_doc;
                }
                memcpy(context->buffer_write_ptr_doc, context->buffer_encode, encoded_size);
                context->buffer_write_ptr_doc += encoded_size;
                context->term32_entity->index.size_encoded_doc += encoded_size;
                context->term32_entity->index.num_encoded_doc += context->num_doc;
                
                //write(context->fd_doc, context->buffer_write, encoded_size);
                context->doc_ptr = context->buffer_doc;
                //if( encoded_size > context->num_doc * sizeof(uint32_t)) printf("error encoding doc ids %d, %d\n", encoded_size,context->num_doc * sizeof(uint32_t) );
                context->num_doc = 0;
                
            }
            //add new doc_id;
            *context->doc_ptr++ = term32->document_id - context->last_document_id;
            context->num_doc++;
            context->num_docid++;
            context->last_document_id = term32->document_id;
            /*            
            if(context->last_field_id) {
                uint8_t field_id = tyn_lget_array_idx_uint32_t(term32->field_id, context->fulltext_field_hashes, context->num_fulltext_fields);
                *context->freq_ptr = *context->freq_ptr << 8 | field_id;
                context->freq_ptr++;
                context->num_freq++;
                //*context->freq_ptr = 0;
            }
            */
            context->last_field_id = 0;
        }
        if(term32->field_id != context->last_field_id) {
            if(context->num_freq == context->num_max_freq) {
                size_t encoded_size = tyn_p4d_encode32(context->buffer_freq, context->num_freq, context->buffer_encode, 0);
                if(context->buffer_write_ptr_freq + encoded_size > context->buffer_write_max_freq) {
                    //flush to disk
                    tyn_write(context->fd_freq, context->buffer_write_freq, context->buffer_write_ptr_freq - context->buffer_write_freq);
                    context->buffer_write_ptr_freq = context->buffer_write_freq;
                }
                memcpy(context->buffer_write_ptr_freq, context->buffer_encode, encoded_size);
                context->buffer_write_ptr_freq += encoded_size;
                context->term32_entity->index.size_encoded_freq += encoded_size;
                context->term32_entity->index.num_encoded_freq += context->num_freq;
                
                //write(context->fd_freq, context->buffer_write, encoded_size);
                memset(context->buffer_freq, 0, context->num_freq * sizeof(uint32_t));
                context->freq_ptr = context->buffer_freq;
                context->num_freq = 0;
                context->last_field_id = 0;
            }
            //if(context->last_field_id) {
                uint8_t field_id = tyn_lget_array_idx_uint32_t(term32->field_id, context->fulltext_field_hashes, context->num_fulltext_fields);
                *context->freq_ptr = *context->freq_ptr << 8 | field_id;
                context->freq_ptr++;
                context->num_freq++;
                //*context->freq_ptr = 0;
            //}
            
            context->last_field_id = term32->field_id;
            
        }
        (*context->freq_ptr)++;
        
        *context->pos_ptr++ = term32->pos;
        context->num_pos++;
        
        

        if(context->num_pos == context->num_max_pos) {
            size_t encoded_size = tyn_p4d_encode32(context->buffer_pos, context->num_pos, context->buffer_encode, 0);
            if(context->buffer_write_ptr_pos + encoded_size > context->buffer_write_max_pos) {
                //flush to disk
                tyn_write(context->fd_pos, context->buffer_write_pos, context->buffer_write_ptr_pos - context->buffer_write_pos);
                context->buffer_write_ptr_pos = context->buffer_write_pos;
            }
            memcpy(context->buffer_write_ptr_pos, context->buffer_encode, encoded_size);
            context->buffer_write_ptr_pos += encoded_size;
            context->term32_entity->index.size_encoded_pos += encoded_size;
            context->term32_entity->index.num_encoded_pos += context->num_pos;
            
            //write(context->fd_pos, context->buffer_write, encoded_size);
            context->pos_ptr = context->buffer_pos;
            context->num_pos = 0;
        }
        term32++;
        counter++;
        //if(context->is_first) {
        //    context->is_first = 0;
        //}
        //printf("offset %u %u %u\n", context->term32_entity->index.offset_doc,context->term32_entity->index.offset_freq,context->term32_entity->index.offset_pos);
        
    }
    if(is_last) {
        if(context->num_doc) {
            size_t encoded_size = tyn_p4d_encode32(context->buffer_doc, context->num_doc, context->buffer_encode, 1);
            memcpy(context->buffer_write_ptr_doc, context->buffer_encode, encoded_size);
            context->buffer_write_ptr_doc += encoded_size;
            context->term32_entity->index.size_encoded_doc += encoded_size;
            context->term32_entity->index.num_encoded_doc += context->num_doc;

            //if(context->buffer_write_ptr_doc + encoded_size > context->buffer_write_max_doc) {
                //flush to disk
                tyn_write(context->fd_doc, context->buffer_write_doc, context->buffer_write_ptr_doc - context->buffer_write_doc);
                context->buffer_write_ptr_doc = context->buffer_write_doc;
            //}

            //write(context->fd_doc, context->buffer_write, encoded_size);
            context->doc_ptr = context->buffer_doc;
            context->num_doc = 0;
        }
        if(context->num_freq) {
            size_t encoded_size = tyn_p4d_encode32(context->buffer_freq, context->num_freq, context->buffer_encode, 1);
            memcpy(context->buffer_write_ptr_freq, context->buffer_encode, encoded_size);
            context->buffer_write_ptr_freq += encoded_size;
            context->term32_entity->index.size_encoded_freq += encoded_size;
            context->term32_entity->index.num_encoded_freq += context->num_freq;

            //if(context->buffer_write_ptr_freq + encoded_size > context->buffer_write_max_freq) {
                //flush to disk
                tyn_write(context->fd_freq, context->buffer_write_freq, context->buffer_write_ptr_freq - context->buffer_write_freq);
                context->buffer_write_ptr_freq = context->buffer_write_freq;
            //}

            //write(context->fd_freq, context->buffer_write, encoded_size);
            memset(context->buffer_freq, 0, context->num_freq * sizeof(uint32_t));    
            context->freq_ptr = context->buffer_freq;
            context->num_freq = 0;
            //memset(context->buffer_freq, 0, context->num_freq * sizeof(uint32_t));
        }
        if(context->num_pos) {
            size_t encoded_size = tyn_p4d_encode32(context->buffer_pos, context->num_pos, context->buffer_encode, 1);
            memcpy(context->buffer_write_ptr_pos, context->buffer_encode, encoded_size);
            context->buffer_write_ptr_pos += encoded_size;
            context->term32_entity->index.size_encoded_pos += encoded_size;
            context->term32_entity->index.num_encoded_pos += context->num_pos;

            //if(context->buffer_write_ptr_pos + encoded_size > context->buffer_write_max_pos) {
                //flush to disk
                tyn_write(context->fd_pos, context->buffer_write_pos, context->buffer_write_ptr_pos - context->buffer_write_pos);
                context->buffer_write_ptr_pos = context->buffer_write_pos;
            //}

            //write(context->fd_pos, context->buffer_write, encoded_size);
            context->pos_ptr = context->buffer_pos;
            context->num_pos = 0;
        }
    }
    //write(((term32_context_t *)nodes_context)->fd_terms32, array, size);
}

int nodes_attr32_flush(void *array, size_t size, void *nodes_context) {
    attr32_context_t *context = (attr32_context_t *)nodes_context;
    char *buffer_cur = context->buffer;
    
    attr32_t *attr32 = (attr32_t *)array;
    size_t total = size / sizeof(attr32_t);
    
    size_t counter = 0;
    
    while(counter < total) {
        memcpy(buffer_cur, &(attr32->document_id), sizeof(uint32_t));
        buffer_cur += sizeof(uint32_t);
        pread(context->fd_attrs32_unsorted, buffer_cur, context->attrs_length, attr32->unit_idx * context->attrs_length);
        buffer_cur += context->attrs_length;
        if(buffer_cur + sizeof(uint32_t) + context->attrs_length > context->buffer + context->buffer_size) {
            tyn_write(context->fd_attrs32, context->buffer, buffer_cur - context->buffer);
            buffer_cur = context->buffer;
        }
        attr32++;
        counter++;
    }
    if(buffer_cur > context->buffer) {
        tyn_write(context->fd_attrs32, context->buffer, buffer_cur - context->buffer);
    }
}


int tyn_build_index(tyn_config_t *tyn_config, char *index_name) {
    size_t memory_limit = tyn_config->indexer->memory_limit;
    printf("memory limit:%d\n", memory_limit);
    for(int i = 0; i < tyn_config->indexes_count; i++) {
        tyn_config_index_t *tyn_config_index = *(tyn_config->indexes + i);
        if(index_name) {
            if(strcmp(tyn_config_index->index_name, index_name) != 0) {
                continue;
            }
        }
        clock_t cl;
        cl = clock();
        
        //load source dynamic library
        void *dl_reader = dlopen(tyn_config_index->reader_path, RTLD_LAZY);
        void *dl_tokenizer = dlopen(tyn_config_index->tokenizer_path, RTLD_LAZY);
        
        if (!dl_reader) {
            tyn_log(stderr, "'%s' failed to load!", tyn_config_index->reader_path);
            tyn_log(stderr, dlerror());
        }
        if (!dl_tokenizer) {
            tyn_log(stderr, "'%s' failed to load!", tyn_config_index->tokenizer_path);
            tyn_log(stderr, dlerror());
        }

        //the function's prototype
        int (* tyn_reader_create)(config_setting_t *, tyn_reader_t *);
        int (* tyn_reader_destroy)(tyn_reader_t *);
        tyn_document_t* (* tyn_reader_fetch_document)(tyn_reader_t *);
        int (* tyn_tokenizer_create)(config_setting_t *, tyn_tokenizer_t *);
        int (* tyn_tokenizer_destroy)(tyn_tokenizer_t *);
        int (* tyn_tokenizer_set_text)(char *, size_t, tyn_tokenizer_t *);
        tyn_token_t * (* tyn_tokenizer_get_token)(tyn_tokenizer_t *);
        
        //initialize the reader create function
        tyn_reader_create = (int (*)(config_setting_t *, tyn_reader_t *)) dlsym(dl_reader, "tyn_reader_create");
        tyn_reader_destroy = (int (*)(tyn_reader_t *)) dlsym(dl_reader, "tyn_reader_destroy");
        tyn_reader_fetch_document = (int (*)(tyn_reader_t *)) dlsym(dl_reader, "tyn_reader_fetch_document");
        tyn_tokenizer_create = (int (*)(config_setting_t *, tyn_tokenizer_t *))dlsym(dl_tokenizer, "tyn_tokenizer_create");
        tyn_tokenizer_destroy = (int (*)(tyn_tokenizer_t *))dlsym(dl_tokenizer, "tyn_tokenizer_destroy");
        tyn_tokenizer_set_text = (int (*)(char *, size_t, tyn_tokenizer_t *))dlsym(dl_tokenizer, "tyn_tokenizer_set_text");
        tyn_tokenizer_get_token = (int (*)(tyn_tokenizer_t *))dlsym(dl_tokenizer, "tyn_tokenizer_get_token");
        
        if(tyn_reader_create == NULL || tyn_reader_destroy == NULL || tyn_reader_fetch_document == NULL) {
            tyn_log(stderr, "'%s': failed to load reader functions", tyn_config_index->reader_path);
            tyn_log(stderr, "%s", dlerror());
            return TYN_ERROR;
        }
        if(tyn_tokenizer_create == NULL || tyn_tokenizer_destroy == NULL || tyn_tokenizer_set_text == NULL || tyn_tokenizer_get_token == NULL) {
            tyn_log(stderr, "'%s': failed to load tokenizer functions", tyn_config_index->tokenizer_path);
            tyn_log(stderr, "%s", dlerror());
            return TYN_ERROR;
        }

        tyn_reader_t *reader = (tyn_reader_t *)malloc( sizeof(tyn_reader_t) );
        (*tyn_reader_create)(tyn_config_index->reader_params, reader);
        
        tyn_tokenizer_t *tokenizer = (tyn_tokenizer_t *) malloc( sizeof(tyn_tokenizer_t) );
        (*tyn_tokenizer_create)(tyn_config_index->tokenizer_params, tokenizer);
        
        tyn_document_t *document;
        size_t bytes_collected = 0;
        
        //prepare the memory buffers
        size_t buffer_size= 8*1024*1024;
        //fulltext, attr_int8, attr_int16, attr_int32, attr_int64, attr_float, attr32_timestamp
        
        
        size_t attrs_length;
        
        attrs_length = tyn_config_index->num_attr_int8_t_fields * sizeof(int8_t);
        attrs_length += tyn_config_index->num_attr_int16_t_fields * sizeof(int16_t);
        attrs_length += tyn_config_index->num_attr_int32_t_fields * sizeof(int32_t);
        attrs_length += tyn_config_index->num_attr_int64_t_fields * sizeof(int64_t);
        attrs_length += tyn_config_index->num_attr_uint8_t_fields * sizeof(uint8_t);
        attrs_length += tyn_config_index->num_attr_uint16_t_fields * sizeof(uint16_t);
        attrs_length += tyn_config_index->num_attr_uint32_t_fields * sizeof(uint32_t);
        attrs_length += tyn_config_index->num_attr_uint64_t_fields * sizeof(uint64_t);
        attrs_length += tyn_config_index->num_attr_float_fields * sizeof(float);
        attrs_length += tyn_config_index->num_attr_double_fields * sizeof(double);
        //attrs_length += tyn_config->num_attr_int16_t_fields * sizeof(int16_t);
        
        size_t attrs_buffer_size = 1*1024*1024;
        size_t max_num_attrs = attrs_buffer_size / attrs_length;
        printf("max num attrs: %d\n", max_num_attrs);
        char* const attrs = (char *)malloc( attrs_length * max_num_attrs);
        char *p_attrs = attrs; 
        size_t num_attrs = 0;
        
        //initialize field idx cache
        int *attr32_int8_t_field_idxs = (int *)malloc( sizeof(int) * tyn_config_index->num_attr_int8_t_fields );
        memset(attr32_int8_t_field_idxs, 0, sizeof(int) * tyn_config_index->num_attr_int8_t_fields);
        int *attr32_int16_t_field_idxs = (int *)malloc( sizeof(int) * tyn_config_index->num_attr_int16_t_fields );
        memset(attr32_int16_t_field_idxs, 0, sizeof(int) * tyn_config_index->num_attr_int16_t_fields);
        int *attr32_int32_t_field_idxs = (int *)malloc( sizeof(int) * tyn_config_index->num_attr_int32_t_fields );
        memset(attr32_int32_t_field_idxs, 0, sizeof(int) * tyn_config_index->num_attr_int32_t_fields);
        int *attr32_int64_t_field_idxs = (int *)malloc( sizeof(int) * tyn_config_index->num_attr_int64_t_fields );
        memset(attr32_int64_t_field_idxs, 0, sizeof(int) * tyn_config_index->num_attr_int64_t_fields);
        
        int *attr32_uint8_t_field_idxs = (int *)malloc( sizeof(int) * tyn_config_index->num_attr_uint8_t_fields );
        memset(attr32_uint8_t_field_idxs, 0, sizeof(int) * tyn_config_index->num_attr_uint8_t_fields);
        int *attr32_uint16_t_field_idxs = (int *)malloc( sizeof(int) * tyn_config_index->num_attr_uint16_t_fields );
        memset(attr32_uint16_t_field_idxs, 0, sizeof(int) * tyn_config_index->num_attr_uint16_t_fields);
        int *attr32_uint32_t_field_idxs = (int *)malloc( sizeof(int) * tyn_config_index->num_attr_uint32_t_fields );
        memset(attr32_uint32_t_field_idxs, 0, sizeof(int) * tyn_config_index->num_attr_uint32_t_fields);
        int *attr32_uint64_t_field_idxs = (int *)malloc( sizeof(int) * tyn_config_index->num_attr_uint64_t_fields );
        memset(attr32_uint64_t_field_idxs, 0, sizeof(int) * tyn_config_index->num_attr_uint64_t_fields);
        
        int *attr32_float_field_idxs = (int *)malloc( sizeof(int) * tyn_config_index->num_attr_float_fields );
        memset(attr32_float_field_idxs, 0, sizeof(int) * tyn_config_index->num_attr_float_fields);
        int *attr32_double_field_idxs = (int *)malloc( sizeof(int) * tyn_config_index->num_attr_double_fields );
        memset(attr32_double_field_idxs, 0, sizeof(int) * tyn_config_index->num_attr_double_fields);

        int *fulltext_field_idxs = (int *)malloc( sizeof(int) * tyn_config_index->num_fulltext_fields );
        memset(fulltext_field_idxs, 0, sizeof(int) * tyn_config_index->num_fulltext_fields);


        char fname_attrs32_idx[256];
        sprintf(fname_attrs32_idx, "%s.tya.idx", tyn_config_index->index_name);
        int fd_attrs32_idx = open(fname_attrs32_idx, O_CREAT|O_RDWR|O_TRUNC, S_IRWXU|S_IRWXG);
        size_t attrs32_buffer_size = 64*1024*1024;
        char *attrs32_buffer = (char *)malloc( attrs32_buffer_size );
        tyn_exsorter_t *exsorter_attrs = tyn_exsorter_create(attrs32_buffer, attrs32_buffer_size, sizeof(attr32_t), tyn_sort_attr32, tyn_cmp_node_attr32, fd_attrs32_idx);

        char fname_attrs32_unsorted[256];
        sprintf(fname_attrs32_unsorted, "%s.tya.dat", tyn_config_index->index_name);
        int fd_attrs32_unsorted = open(fname_attrs32_unsorted, O_CREAT|O_RDWR|O_TRUNC, S_IRWXU|S_IRWXG);
        
        char fname_hits32_data[256];
        sprintf(fname_hits32_data, "%s.tyd.dat", tyn_config_index->index_name);
        int fd_hits32_data = open(fname_hits32_data, O_CREAT|O_RDWR|O_TRUNC, S_IRWXU|S_IRWXG);
        size_t terms32_buffer_size = 64*1024*1024;
        char *terms32_buffer = (char *)malloc( terms32_buffer_size );
        tyn_exsorter_t *exsorter_terms = tyn_exsorter_create(terms32_buffer, terms32_buffer_size, sizeof(term32_t), tyn_sort_term32, tyn_cmp_node_term32, fd_hits32_data);

        term32_t *term32 = (term32_t *)malloc( sizeof(term32_t) );
        attr32_t *attr32 = (attr32_t *)malloc( sizeof(attr32_t) );
        
        size_t documents_collected = 0;
        size_t attr_unit_idx = 0;
        term32_entity_t *term32_entities_by_id = NULL;
        term32_entity_t *term32_entities_by_name = NULL;
        uint32_t max_term32_entity = 1;
        while( (document = (*tyn_reader_fetch_document)(reader)) != NULL) {
            uint32_t document_id = 0;
            //first we get document id
            char id_string[1024];//fix this if length > 1024?
            uint32_t field_hashes[1024];//fix this if number > 1024?
            for(int i = 0; i < document->num_fields; i++) {
                field_hashes[i] = tyn_hash32(*(document->field_names + i), *(document->field_name_lengths + i));
                if(field_hashes[i] == tyn_config_index->doc_id_field_hash) {
                    document_id = tyn_strntoi(*(document->fields + i), *(document->field_lengths + i));
                }
            }
            //qsort(field_hashes, document->num_fields, sizeof(uint32_t), tyn_compare_uint32_t);
            if(document_id == 0) {
                tyn_log(stderr, "\ndocument id not found, contiue\n");
                continue;
            }

            documents_collected++;
            if(documents_collected %1000 == 0) {
                printf("document collected: %d\r", documents_collected);
                fflush(stdout);
            }
            

            if(tyn_config_index->num_fulltext_fields) {
                for( int i = 0; i < tyn_config_index->num_fulltext_fields; i++) {
                    //int idx = tyn_bget_array_idx_uint32_t(*(tyn_config_index->fulltext_field_hashes + i), field_hashes, document->num_fields);
                    if( *(field_hashes + *(fulltext_field_idxs + i)) != *(tyn_config_index->fulltext_field_hashes + i)) {
                        printf("not hits\n");
                        *(fulltext_field_idxs + i) = tyn_lget_array_idx_uint32_t(*(tyn_config_index->fulltext_field_hashes + i), field_hashes, document->num_fields);
                    } 
                    int idx = *(fulltext_field_idxs + i);
                    if(idx >= 0) {
                        bytes_collected += document->field_lengths[idx];
                        (*tyn_tokenizer_set_text)(document->fields[idx], document->field_lengths[idx], tokenizer);
                        tyn_token_t *token;
                        while((token = (*tyn_tokenizer_get_token)(tokenizer)) != NULL) {
                            uint32_t word_id;
                            term32_entity_t *term32_entity = NULL;
                            HASH_FIND(hh,term32_entities_by_name,token->str,token->str_len,term32_entity);
                            if(term32_entity) {
                                //this token has already been assigned id
                                word_id = term32_entity->id;
                            } else {
                                char *str = (char *)malloc( token->str_len + 1);
                                *(str + token->str_len) = '\0';
                                memcpy(str, token->str, token->str_len);
                                term32_entity = (term32_entity_t *)malloc( sizeof(term32_entity_t) );
                                term32_entity->str = str;
                                term32_entity->str_len = token->str_len;
                                word_id = max_term32_entity++;
                                term32_entity->id = word_id;
                                HASH_ADD_KEYPTR( hh, term32_entities_by_name, term32_entity->str, term32_entity->str_len, term32_entity );
                                HASH_ADD(hh2, term32_entities_by_id, id, sizeof(uint32_t), term32_entity);
                            }
                            
                            //term32->word_id = tyn_hash32(token->str, token->str_len);
                            term32->word_id = word_id;
                            term32->document_id = document_id;
                            term32->field_id = tyn_hash32(*(document->field_names + idx), *(document->field_name_lengths + idx));
                            term32->pos = token->pos;
                            //printf("w:%u\t d:%u\t f:%u\t p:%u\n", term32->word_id, term32->document_id, term32->field_id, term32->pos);
                            tyn_exsorter_add(term32, exsorter_terms);

                        }
                    } else {
                        tyn_log(stderr, "\ndocument id: %d, can not find field '%s'", document_id, *(tyn_config_index->fulltext_fields + i) );
                        return TYN_ERROR;
                    }
                }
            }

#define TYN_ATTR_ACCUMULATE(BITS, TYPE, CONV_FUNC) \
    if(tyn_config_index->num_attr_##TYPE##_fields) { \
        for( int i = 0; i < tyn_config_index->num_attr_##TYPE##_fields; i++) { \
            if( *(field_hashes + *(attr##BITS##_##TYPE##_field_idxs + i)) !=  *(tyn_config_index->attr_##TYPE##_field_hashes + i) ) {\
                 *(attr##BITS##_##TYPE##_field_idxs + i) = tyn_lget_array_idx_uint32_t(*(tyn_config_index->attr_##TYPE##_field_hashes + i), field_hashes, document->num_fields); \
            } \
            int idx =  *(attr##BITS##_##TYPE##_field_idxs + i); \
            if(idx >= 0) { \
                bytes_collected += document->field_lengths[idx]; \
                TYPE attr = (TYPE)CONV_FUNC(document->fields[idx], document->field_lengths[idx]); \
                memcpy(p_attrs, &attr, sizeof(TYPE)); \
                p_attrs += sizeof(TYPE); \
            } else { \
                tyn_log(stderr, "document id: %d, can not find field '%s'", document_id, *(tyn_config_index->attr_##TYPE##_fields + i) ); \
                return TYN_ERROR; \
            } \
        } \
    }

            TYN_ATTR_ACCUMULATE(32, int8_t, tyn_strntoi);
            TYN_ATTR_ACCUMULATE(32, int16_t, tyn_strntoi);
            TYN_ATTR_ACCUMULATE(32, int32_t, tyn_strntoi);
            TYN_ATTR_ACCUMULATE(32, int64_t, tyn_strntol);
            
            TYN_ATTR_ACCUMULATE(32, uint8_t, tyn_strntoui);
            TYN_ATTR_ACCUMULATE(32, uint16_t, tyn_strntoui);
            TYN_ATTR_ACCUMULATE(32, uint32_t, tyn_strntoui);
            TYN_ATTR_ACCUMULATE(32, uint64_t, tyn_strntoul);
            
            TYN_ATTR_ACCUMULATE(32, float, tyn_strntof);
            TYN_ATTR_ACCUMULATE(32, double, tyn_strntod);
            num_attrs++;
            
            if( num_attrs == max_num_attrs) {
                tyn_write(fd_attrs32_unsorted, attrs, num_attrs * attrs_length);
                //fwrite(attrs, 1, attrs_length, fp_attrs32_unsorted);
                num_attrs = 0;
                p_attrs = attrs;
            }
            //attr32_t *attr32 = (attr32_t *)malloc( sizeof(attr32_t) );
            attr32->document_id = document_id;
            attr32->unit_idx = attr_unit_idx++;
            tyn_exsorter_add(attr32, exsorter_attrs);
        }
        if( num_attrs > 0 ) {
            tyn_write(fd_attrs32_unsorted, attrs, num_attrs * attrs_length);
            //fwrite(attrs, 1, attrs_length, fp_attrs32_unsorted);
            num_attrs = 0;
            p_attrs = attrs;
        }
        

        printf("\n");
        printf("sorting hits...\n");
        //int fd_write = open("tyn_hits.tyt", O_CREAT|O_RDWR|O_TRUNC, S_IRWXU|S_IRWXG);
        char fname_doc[256];
        char fname_freq[256];
        char fname_pos[256];
        sprintf(fname_doc, "%s.tyd", tyn_config_index->index_name);
        sprintf(fname_freq, "%s.tyf", tyn_config_index->index_name);
        sprintf(fname_pos, "%s.typ", tyn_config_index->index_name);
        int fd_doc = open(fname_doc, O_CREAT|O_RDWR|O_TRUNC, S_IRWXU|S_IRWXG);
        int fd_freq = open(fname_freq, O_CREAT|O_RDWR|O_TRUNC, S_IRWXU|S_IRWXG);
        int fd_pos = open(fname_pos, O_CREAT|O_RDWR|O_TRUNC, S_IRWXU|S_IRWXG);
        size_t size_buffer_doc = 1*1024*1024;
        size_t size_buffer_freq = 1*1024*1024;
        size_t size_buffer_pos = 1*1024*1024;
        size_t size_buffer_encode = 2*1024*1024;
        size_t size_buffer_write_doc = 4*1024*1024;
        size_t size_buffer_write_freq = 4*1024*1024;
        size_t size_buffer_write_pos = 4*1024*1024;
        char *buffer_doc = (char *)malloc( size_buffer_doc );
        char *buffer_freq = (char *)malloc( size_buffer_freq );
        char *buffer_pos = (char *)malloc( size_buffer_pos );
        char *buffer_encode = (char *)malloc( size_buffer_encode);
        char *buffer_write_doc = (char *)malloc( size_buffer_write_doc);
        char *buffer_write_freq = (char *)malloc( size_buffer_write_freq);
        char *buffer_write_pos = (char *)malloc( size_buffer_write_pos);
        
        term32_context_t *term32_context = (term32_context_t *)malloc( sizeof(term32_context_t) );
        term32_context->term32_entities = term32_entities_by_id;
        //term32_context->fd_terms32 = fd_write;
        
        term32_context->is_first = 1;
        term32_context->field_idx = 0;
        
        term32_context->last_word_id = 0;
        term32_context->last_document_id = 0;
        term32_context->last_field_id = 0;
        
        term32_context->fd_doc = fd_doc;
        term32_context->fd_freq = fd_freq;
        term32_context->fd_pos = fd_pos;
        
        term32_context->size_buffer_doc = size_buffer_doc;
        term32_context->size_buffer_freq = size_buffer_freq;
        term32_context->size_buffer_pos = size_buffer_pos;
        term32_context->size_buffer_encode = size_buffer_encode;
        term32_context->size_buffer_write_doc = size_buffer_write_doc;
        term32_context->size_buffer_write_freq = size_buffer_write_freq;
        term32_context->size_buffer_write_pos = size_buffer_write_pos;
        
        term32_context->buffer_doc = buffer_doc;
        term32_context->buffer_freq = buffer_freq;
        term32_context->buffer_pos = buffer_pos;
        term32_context->buffer_encode = buffer_encode;
        term32_context->buffer_write_doc = buffer_write_doc;
        term32_context->buffer_write_freq = buffer_write_freq;
        term32_context->buffer_write_pos = buffer_write_pos;
        term32_context->buffer_write_ptr_doc = term32_context->buffer_write_doc;
        term32_context->buffer_write_ptr_freq = term32_context->buffer_write_freq;
        term32_context->buffer_write_ptr_pos = term32_context->buffer_write_pos;
        term32_context->buffer_write_max_doc = term32_context->buffer_write_doc + term32_context->size_buffer_write_doc;
        term32_context->buffer_write_max_freq = term32_context->buffer_write_freq + term32_context->size_buffer_write_freq;
        term32_context->buffer_write_max_pos = term32_context->buffer_write_pos + term32_context->size_buffer_write_pos;
        
        memset(term32_context->buffer_freq, 0, term32_context->size_buffer_freq);
        
        term32_context->fulltext_field_hashes = tyn_config_index->fulltext_field_hashes;
        term32_context->num_fulltext_fields = tyn_config_index->num_fulltext_fields;
        
        term32_context->num_doc = 0;
        term32_context->num_freq = 0;
        term32_context->num_pos = 0;
        
        term32_context->num_docid = 0;
        term32_context->num_wordid= 0;
        
        term32_context->num_max_doc = size_buffer_doc / sizeof(uint32_t) / 128  * 128; //make sure it's times of 128
        term32_context->num_max_freq = size_buffer_freq / sizeof(uint32_t) / 128 * 128;
        term32_context->num_max_pos = size_buffer_pos / sizeof(uint32_t) / 128 * 128;
        
        term32_context->doc_ptr = term32_context->buffer_doc;
        term32_context->freq_ptr = term32_context->buffer_freq;
        term32_context->pos_ptr = term32_context->buffer_pos;
        
        //term32_context->offset_doc = 0;
        //term32_context->offset_freq = 0;
        //term32_context->offset_pos = 0;
        
        tyn_exsorter_sort(exsorter_terms, term32_context, nodes_term32_flush, progress_callback);
        printf("total docid %d\n", term32_context->num_docid);
        printf("total wordid %d\n", term32_context->num_wordid);
        //------------------------------------------------------------------------------------------------------------
        printf("sorting attrs...\n");
        char fname_attrs32[256];
        sprintf(fname_attrs32, "%s.tya", tyn_config_index->index_name);
        int fd_attrs32 = open(fname_attrs32, O_CREAT|O_RDWR|O_TRUNC, S_IRWXU|S_IRWXG);

        attr32_context_t *attr32_context = (attr32_context_t *)malloc( sizeof(attr32_context_t) );
        attr32_context->fd_attrs32_unsorted = fd_attrs32_unsorted;
        attr32_context->attrs_length = attrs_length;
        attr32_context->fd_attrs32 = fd_attrs32;
        
        size_t attrs_write_buffer_size = 16*1024*1024;//adjust parameter
        attr32_context->buffer_size = attrs_write_buffer_size;
        attr32_context->buffer = (char *)malloc( attrs_write_buffer_size );
        //exsorter_attrs->output_buffer_size = 16*1024*1024;
        tyn_exsorter_sort(exsorter_attrs, attr32_context, nodes_attr32_flush, progress_callback);

        
        //-----------------------------------------------------------------------------------
        //write term index
        char fname_term32_index[256];
        sprintf(fname_term32_index, "%s.tyi", tyn_config_index->index_name);
        int fd_term32_index = open(fname_term32_index, O_CREAT|O_RDWR|O_TRUNC, S_IRWXU|S_IRWXG);
        size_t term32_index_write_buffer_size = 8*1024*1024;
        char *term32_index_write_buffer = (char *)malloc(term32_index_write_buffer_size);
        char *term32_index_write_buffer_ptr = term32_index_write_buffer;
        char *term32_index_write_buffer_max_ptr = term32_index_write_buffer_ptr + term32_index_write_buffer_size;
        char *term32_index_encode_buffer = (char *)malloc(256);
        printf("total word: %d\n", HASH_COUNT(term32_entities_by_name));
        printf("size of index entry: %d\n", sizeof(term32_index_entry_t));
        
        //HASH_SRT(hh, term32_entities_by_name, tyn_term32_entity_sort_by_name);
        size_t size_total_num = 0;
        term32_entity_t *t, *tmp;
        HASH_ITER(hh, term32_entities_by_name, t, tmp) {
            //printf("%s\n", t->str);
            if(term32_index_write_buffer_ptr + sizeof(uint32_t)+ sizeof(term32_index_entry_t)*4 + t->str_len + sizeof(uint8_t) > term32_index_write_buffer_max_ptr) {
                tyn_write(fd_term32_index, term32_index_write_buffer, term32_index_write_buffer_ptr - term32_index_write_buffer);
                term32_index_write_buffer_ptr = term32_index_write_buffer;
            }
            memcpy(term32_index_write_buffer_ptr, &t->str_len, sizeof(uint32_t));
            term32_index_write_buffer_ptr += sizeof(uint32_t);
            memcpy(term32_index_write_buffer_ptr, t->str, t->str_len);
            term32_index_write_buffer_ptr += t->str_len;
            uint8_t encoded_size = tyn_iS16_encode32(&t->index, sizeof(term32_index_entry_t) / sizeof(uint32_t), term32_index_encode_buffer);
            memcpy(term32_index_write_buffer_ptr, &encoded_size, sizeof(uint8_t));
            term32_index_write_buffer_ptr += sizeof(uint8_t);
            memcpy(term32_index_write_buffer_ptr, term32_index_encode_buffer, encoded_size);
            term32_index_write_buffer_ptr += encoded_size;
            /*
            memcpy(term32_index_write_buffer_ptr, &t->index, sizeof(term32_index_entry_t));
            term32_index_write_buffer_ptr += sizeof(term32_index_entry_t);
            */
            size_total_num++;
        }
        //flush the last
        tyn_write(fd_term32_index, term32_index_write_buffer, term32_index_write_buffer_ptr - term32_index_write_buffer);
        term32_index_write_buffer_ptr = term32_index_write_buffer;
        printf("size_total_num:%d\n", size_total_num);
        //------------------------------------------------------------------------------------------
/*
        fname_attrs32_idx
        fname_attrs32_unsorted
        fname_hits32_data
        fname_doc
        fname_freq
        fname_pos
        fname_attrs32
        fname_term32_index
*/
        close(fd_attrs32_idx);
        close(fd_attrs32_unsorted);
        close(fd_hits32_data);
        close(fd_doc);
        close(fd_freq);
        close(fd_pos);
        close(fd_attrs32);
        close(fd_term32_index);

        /*
         * header format
         * uint64_t header_id;
         * uint32_t num_fields;
         * 
         * uint32_t num_fulltext_fields;
         * uint32_t *hashes_fulltext_fields;
         * 
         * uint32_t num_int8_t_fields;
         * uint32_t *hashes_int8_t_fields;
         * 
         * uint32_t num_int16_t_fields;
         * uint32_t *hashes_
         * 
         * 
         */
        //--------------------------------------------------------------------------------------------
        //generate header
        char fname_header[256];
        sprintf(fname_header, "%s.tyh", tyn_config_index->index_name);
        int fd_header = open(fname_header, O_CREAT|O_RDWR|O_TRUNC, S_IRWXU|S_IRWXG);
        
        tyn_header_t tyn_header;
        
        tyn_header.signature = TYN_TYH_SIGN;
        tyn_header.version = 20120321;
        tyn_header.config_file_size = tyn_config->file_str_len;
        tyn_header.total_documents = documents_collected;
        tyn_header.total_bytes = bytes_collected;
        tyn_header.time_created = time(NULL);
        
        tyn_write(fd_header, &tyn_header, sizeof(tyn_header_t));
        lseek(fd_header, 1024, SEEK_SET);
        tyn_write(fd_header, tyn_config->file_str, tyn_config->file_str_len);
        close(fd_header);
        
/*        
#define TYN_HEADER_ACCUMULATE(TYPE) \
        printf("%s: %d\n", #TYPE,tyn_config_index->num_##TYPE##_fields); \
        for(int i = 0; i< tyn_config_index->num_##TYPE##_fields; i++) { \
            printf("\t%u, %s\n", *(tyn_config_index->TYPE##_field_hashes + i), *(tyn_config_index->TYPE##_fields + i)); \
        } \
        
        TYN_HEADER_ACCUMULATE(fulltext)
        TYN_HEADER_ACCUMULATE(attr_int8_t)
        TYN_HEADER_ACCUMULATE(attr_int16_t)
        TYN_HEADER_ACCUMULATE(attr_int32_t)
        TYN_HEADER_ACCUMULATE(attr_int64_t)
        TYN_HEADER_ACCUMULATE(attr_uint8_t)
        TYN_HEADER_ACCUMULATE(attr_uint16_t)
        TYN_HEADER_ACCUMULATE(attr_uint32_t)
        TYN_HEADER_ACCUMULATE(attr_uint64_t)
        TYN_HEADER_ACCUMULATE(attr_float)
        TYN_HEADER_ACCUMULATE(attr_double)
        TYN_HEADER_ACCUMULATE(attr_timestamp)
        
        
        unsigned char sum[64];
        
        sha1_sum(fname_doc, sum);
        for (i=0; i < 20; i++)
            printf ("%02x", sum[i]);
        printf ("  %s\n", fname_doc);
*/
        
        cl = clock() - cl;
        float seconds = 1.0 * cl / CLOCKS_PER_SEC;
        printf("time took: %.3f sec\n", seconds);
        
        
        printf("verifying attr\n");
        size_t no =2678331;
        char *attr_test = (char *)malloc( attrs_length + sizeof(uint32_t));
        pread(fd_attrs32, attr_test, attrs_length + sizeof(uint32_t), no *(attrs_length + sizeof(uint32_t)));
        
        printf("document id: %u\n" , *((uint32_t *)attr_test));
        attr_test += sizeof(uint32_t);
        
        for( int i = 0; i < tyn_config_index->num_attr_int8_t_fields; i++) {
            printf("int8_t %s: %d\n", tyn_config_index->attr_int8_t_fields[i], *((int8_t *)attr_test));
            attr_test += sizeof(int8_t);
        }
        for( int i = 0; i < tyn_config_index->num_attr_int16_t_fields; i++) {
            printf("int16_t %s: %hu\n", tyn_config_index->attr_int16_t_fields[i], *((int16_t *)attr_test));
            attr_test += sizeof(int16_t);
        }
        for( int i = 0; i < tyn_config_index->num_attr_int32_t_fields; i++) {
            printf("int32_t %s: %d\n", tyn_config_index->attr_int32_t_fields[i], *((int32_t *)attr_test));
            attr_test += sizeof(int32_t);
        }
        for( int i = 0; i < tyn_config_index->num_attr_int64_t_fields; i++) {
            printf("int64_t %s: %lld\n", tyn_config_index->attr_int64_t_fields[i], *((int64_t *)attr_test));
            attr_test += sizeof(int64_t);
        }
        for( int i = 0; i < tyn_config_index->num_attr_float_fields; i++) {
            printf("float %s: %f\n", tyn_config_index->attr_float_fields[i], *((float *)attr_test));
            attr_test += sizeof(float);
        }

        
        //printf("\n");
        printf("bytes_collected: %zu\n", bytes_collected);
        (*tyn_reader_destroy)(reader);
        (*tyn_tokenizer_destroy)(tokenizer);
        dlclose(dl_reader);
        dlclose(dl_tokenizer);
        free(document);
        free(reader);
        free(tokenizer);
        seconds = (int)seconds == 0 ? 1 : seconds;
        printf("%d bytes/s\n", bytes_collected / (int)seconds);
       
    }
    
    
    printf("tyn_config->indexer->dl_directory: %s\n", tyn_config->indexer->dl_directory);
    printf("tyn_config->indexer->memory_limit: %d\n", tyn_config->indexer->memory_limit);   
}


int main(int argc, char **argv) {
    config_t cfg;
    config_init(&cfg);

    char *config_file_name = "tyan.conf";
    tyn_config_t *tyn_config = load_config(config_file_name, &cfg);
    if(tyn_config == TYN_ERROR) {
        tyn_log(stderr, "Failed parsing %s", config_file_name);
        return TYN_ERROR;
    }
    tyn_build_index(tyn_config, "mysql_index");
    
    config_destroy(&cfg);
    
}
